package com.xiaotu.spider.project.pageprocessor.news;

import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.util.*;
import com.xiaotu.spider.AUrlPageProcessor;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;

import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;

/**
 * 网易新闻内容爬取
 */
@Component("News163PageProcessor")
public class News163PageProcessor extends AUrlPageProcessor
{
    private static final Logger LOGGER = LoggerFactory.getLogger(News163PageProcessor.class);

    private static final String[] URLS = new String[]{
            "http://ent.163.com/special/000380VU/newsdata_index%s.js?callback=data_callback",//娱乐
            "http://sports.163.com/special/000587PR/newsdata_n_index%s.js?callback=data_callback",//体育
            "http://money.163.com/special/002557S5/newsdata_idx_index.js?callback=data_callback",//财经
            "http://money.163.com/special/002557S6/newsdata_gp_index%s.js?callback=data_callback",//股票
            "http://mobile.163.com/special/index_datalist%s/?callback=data_callback",//手机
            "http://digi.163.com/special/index_datalist%s/?callback=data_callback",//数码
            "http://lady.163.com/special/00264OOD/data_nd_sense%s.js?callback=data_callback",//女人
            "http://travel.163.com/special/00067VEJ/newsdatas_travel%s.js?callback=data_callback",//旅游
            "http://edu.163.com/special/002987KB/newsdata_edu_hot%s.js?callback=data_callback",//教育
            "http://art.163.com/special/00999815/art_redian_api%s.js?callback=data_callback"//艺术
    };

    private static final String[] TYPES = new String[]{"娱乐", "体育", "财经", "股票", "手机", "数码", "女人", "旅游", "教育", "艺术"};

    private static final String[] PAGE_NO = new String[]{"", "_02", "_03", "_04", "_05", "_06", "_07", "_08", "_09",
            "_10"};

    private String filePath;

    @Override
    public void process(Page page)
    {
        if (page.getUrl().toString().contains("/special/"))
            this.listProcess(page);
        else
            this.detailProcess(page);
    }

    private void listProcess(Page page)
    {
        String json = page.getRawText();
        if (StringUtils.isEmpty(json))
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        json = json.trim();
        if (!json.startsWith("data_callback("))
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        json = json.substring("data_callback(".length(), json.lastIndexOf(")"));
        List<Map<String, Object>> list = GsonUtils.fromJson(json, List.class);
        if (list == null || list.isEmpty())
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        list.stream().filter(map ->
        {
            String url = StringUtil.nullToString(map.get("docurl"));
            String fileName = this.getFileName(page, url);
            boolean isExits = FileUtil.fileExits(fileName);
            if (isExits)
                LOGGER.debug("{} exits", fileName);
            return StringUtils.isNotEmpty(url) && !url.contains("/photoview/") && !isExits;
        }).forEach(map -> PageUtils.addTargetRequest(page, map.get("docurl").toString(), page.getRequest().getExtras
                ()));
    }

    private String getFileName(Page page, String url)
    {
        url = url.substring(url.lastIndexOf("/") + "/".length());
        url = StringUtils.isEmpty(url) ? System.currentTimeMillis() + ".html" : url;
        return filePath + page.getRequest().getExtra("type") + File.separator + url;
    }

    private void detailProcess(Page page)
    {
        String text = page.getHtml().xpath("//div[@class='post_text']").get();
        if (StringUtils.isEmpty(text))
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        text = HtmlUtil.delHTMLTag(text, false);
        String fileName = this.getFileName(page, page.getUrl().toString());
        FileUtil.writeFile(fileName, text);
        LOGGER.debug("{} writed success!", fileName);
    }

    @Override
    protected List<JSONObject> getStartRequestUrlsAndParams()
    {
        List<JSONObject> list = new ArrayList<>();
        for (int i = 0; i < TYPES.length; i++)
        {
            final int idx = i;
            Stream.of(PAGE_NO).forEach(pageNo ->
            {
                JSONObject json = new JSONObject();
                json.put(URL_KEY, String.format(URLS[idx], pageNo));
                json.put("type", TYPES[idx]);
                list.add(json);
            });
        }
        return list;
    }

    @Override
    public void initRunParams(Map<String, Object> map)
    {
        String path = StringUtil.nullToString(map.get("path"));
        if (StringUtils.isEmpty(path))
            throw new SpiderException(SpiderExceptionCode.PageProcessorParameterErrorException);
        this.filePath = path;
    }

    @Override
    protected String getSpiderType()
    {
        return "News163";
    }

    @Override
    public String getPageDataType(Page page)
    {
        return Constants.PageDataType.SEARCH_LIST;
    }

    @Override
    protected String[] getStartRequestUrls()
    {
        return null;
    }
}
